{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('/home/husein/ssd3/multilang/sentences.csv', sep = '\\t', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11355733, 3)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng = df.loc[df[1] == 'eng']\n",
    "eng = eng.iloc[:,-1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1774790"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(eng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2260"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms = df.loc[df[1] == 'zlm']\n",
    "ms = ms.iloc[:,-1].tolist()\n",
    "len(ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19873"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ind = df.loc[df[1] == 'ind']\n",
    "ind = ind.iloc[:,-1].tolist()\n",
    "len(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73388"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmn = df.loc[df[1] == 'cmn']\n",
    "cmn = cmn.iloc[:,-1].tolist()\n",
    "len(cmn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6305"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yue = df.loc[df[1] == 'yue']\n",
    "yue = yue.iloc[:,-1].tolist()\n",
    "len(yue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9479117"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "others = df.loc[~df[1].isin(['zlm', 'ind', 'eng', 'yue', 'cmn'])]\n",
    "others = others.iloc[:,-1].tolist()\n",
    "len(others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "411"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_langs = list(set(df[~df[1].isin(['zlm', 'ind', 'eng', 'yue', 'cmn'])][1].tolist()))\n",
    "len(other_langs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[nan,\n",
       " 'sco',\n",
       " 'diq',\n",
       " 'uzb',\n",
       " 'srp',\n",
       " 'aoz',\n",
       " 'osx',\n",
       " 'igs',\n",
       " 'dan',\n",
       " 'sot',\n",
       " 'nch',\n",
       " 'ido',\n",
       " 'frr',\n",
       " 'oss',\n",
       " 'swg',\n",
       " 'tig',\n",
       " 'bis',\n",
       " 'non',\n",
       " 'vep',\n",
       " 'nys',\n",
       " 'pli',\n",
       " 'gan',\n",
       " 'san',\n",
       " 'ber',\n",
       " 'jpa',\n",
       " 'pnb',\n",
       " 'hbo',\n",
       " 'pfl',\n",
       " 'sdh',\n",
       " 'ckt',\n",
       " 'lzh',\n",
       " 'kas',\n",
       " 'srd',\n",
       " 'hsn',\n",
       " 'ofs',\n",
       " 'nob',\n",
       " 'ltz',\n",
       " 'gom',\n",
       " 'ava',\n",
       " 'zsm',\n",
       " 'fkv',\n",
       " 'swh',\n",
       " 'sah',\n",
       " 'mfa',\n",
       " 'gil',\n",
       " 'umb',\n",
       " 'tet',\n",
       " 'glg',\n",
       " 'grc',\n",
       " 'pcd',\n",
       " 'tsn',\n",
       " 'ita',\n",
       " 'nds',\n",
       " 'afh',\n",
       " 'tam',\n",
       " 'liv',\n",
       " 'rif',\n",
       " 'krc',\n",
       " 'gag',\n",
       " 'arg',\n",
       " 'tpw',\n",
       " 'mvv',\n",
       " 'hif',\n",
       " 'tha',\n",
       " 'bar',\n",
       " 'iba',\n",
       " 'rap',\n",
       " 'lij',\n",
       " 'dws',\n",
       " 'oci',\n",
       " 'toi',\n",
       " 'bvy',\n",
       " 'pes',\n",
       " 'fry',\n",
       " 'ben',\n",
       " 'ood',\n",
       " 'xal',\n",
       " 'sjn',\n",
       " 'crk',\n",
       " 'tkl',\n",
       " 'mni',\n",
       " 'shi',\n",
       " 'rus',\n",
       " 'xho',\n",
       " 'oji',\n",
       " 'xqa',\n",
       " 'brx',\n",
       " 'che',\n",
       " 'thv',\n",
       " 'fij',\n",
       " 'mon',\n",
       " 'bom',\n",
       " 'chg',\n",
       " 'vol',\n",
       " 'apc',\n",
       " 'niu',\n",
       " 'akl',\n",
       " 'cos',\n",
       " 'mal',\n",
       " 'myv',\n",
       " 'scn',\n",
       " 'pau',\n",
       " 'swe',\n",
       " 'lao',\n",
       " 'klj',\n",
       " 'szl',\n",
       " 'mnr',\n",
       " 'tyv',\n",
       " 'lin',\n",
       " 'pan',\n",
       " 'skr',\n",
       " 'tur',\n",
       " 'hat',\n",
       " 'pus',\n",
       " 'tir',\n",
       " 'kjh',\n",
       " 'aze',\n",
       " 'lug',\n",
       " 'zgh',\n",
       " 'kor',\n",
       " 'lad',\n",
       " 'cbk',\n",
       " 'ext',\n",
       " 'bho',\n",
       " 'cyo',\n",
       " 'cmo',\n",
       " 'enm',\n",
       " 'min',\n",
       " 'hak',\n",
       " 'ron',\n",
       " 'tok',\n",
       " 'wol',\n",
       " 'tts',\n",
       " 'nus',\n",
       " 'frm',\n",
       " 'nno',\n",
       " 'nau',\n",
       " 'jam',\n",
       " 'moh',\n",
       " 'swc',\n",
       " 'ukr',\n",
       " 'epo',\n",
       " 'bod',\n",
       " 'mya',\n",
       " 'quc',\n",
       " 'nnb',\n",
       " 'rue',\n",
       " 'lim',\n",
       " 'fuc',\n",
       " 'jbo',\n",
       " 'koi',\n",
       " 'smo',\n",
       " 'kaz',\n",
       " 'mfe',\n",
       " 'grn',\n",
       " 'rhg',\n",
       " 'mkd',\n",
       " 'yua',\n",
       " 'afr',\n",
       " 'hnj',\n",
       " 'kmr',\n",
       " 'tmr',\n",
       " 'ckb',\n",
       " 'tuk',\n",
       " 'hsb',\n",
       " 'wln',\n",
       " 'syc',\n",
       " 'ota',\n",
       " 'abk',\n",
       " 'ell',\n",
       " 'shs',\n",
       " 'bfz',\n",
       " 'lkt',\n",
       " 'ayl',\n",
       " 'lvs',\n",
       " 'aln',\n",
       " 'tat',\n",
       " 'ang',\n",
       " 'vec',\n",
       " 'amh',\n",
       " 'cpi',\n",
       " 'bzt',\n",
       " 'kek',\n",
       " 'nya',\n",
       " 'fur',\n",
       " 'bjn',\n",
       " 'iii',\n",
       " 'yid',\n",
       " 'ina',\n",
       " 'lzz',\n",
       " 'ary',\n",
       " 'hun',\n",
       " 'slv',\n",
       " 'run',\n",
       " 'aii',\n",
       " 'orv',\n",
       " 'arz',\n",
       " 'kab',\n",
       " 'kir',\n",
       " 'bul',\n",
       " 'kam',\n",
       " 'mgm',\n",
       " 'zza',\n",
       " 'lou',\n",
       " 'mad',\n",
       " 'bos',\n",
       " 'khm',\n",
       " 'shy',\n",
       " 'sun',\n",
       " 'ssw',\n",
       " 'phn',\n",
       " 'gcf',\n",
       " 'dtp',\n",
       " 'jpn',\n",
       " 'nov',\n",
       " 'guc',\n",
       " 'tvl',\n",
       " 'egl',\n",
       " 'mic',\n",
       " 'bre',\n",
       " '\\\\N',\n",
       " 'por',\n",
       " 'chr',\n",
       " 'sin',\n",
       " 'vie',\n",
       " 'sqi',\n",
       " 'slk',\n",
       " 'npi',\n",
       " 'nah',\n",
       " 'mai',\n",
       " 'chv',\n",
       " 'dng',\n",
       " 'cor',\n",
       " 'tzl',\n",
       " 'asm',\n",
       " 'qya',\n",
       " 'max',\n",
       " 'hrv',\n",
       " 'pdc',\n",
       " 'ppl',\n",
       " 'jav',\n",
       " 'tmw',\n",
       " 'wuu',\n",
       " 'ile',\n",
       " 'ori',\n",
       " 'tso',\n",
       " 'ldn',\n",
       " 'fin',\n",
       " 'kzj',\n",
       " 'tpi',\n",
       " 'pag',\n",
       " 'mlg',\n",
       " 'lmo',\n",
       " 'jdt',\n",
       " 'ast',\n",
       " 'pal',\n",
       " 'rel',\n",
       " 'isl',\n",
       " 'pms',\n",
       " 'nog',\n",
       " 'acm',\n",
       " 'hax',\n",
       " 'zea',\n",
       " 'new',\n",
       " 'ryu',\n",
       " 'bam',\n",
       " 'pam',\n",
       " 'stq',\n",
       " 'afb',\n",
       " 'ltg',\n",
       " 'mhr',\n",
       " 'alt',\n",
       " 'heb',\n",
       " 'kbd',\n",
       " 'cay',\n",
       " 'hrx',\n",
       " 'krl',\n",
       " 'kha',\n",
       " 'tlh',\n",
       " 'sag',\n",
       " 'urd',\n",
       " 'cycl',\n",
       " 'mlt',\n",
       " 'kaa',\n",
       " 'deu',\n",
       " 'gaa',\n",
       " 'vro',\n",
       " 'awa',\n",
       " 'mar',\n",
       " 'cat',\n",
       " 'uig',\n",
       " 'fao',\n",
       " 'bal',\n",
       " 'pol',\n",
       " 'ilo',\n",
       " 'mnc',\n",
       " 'nav',\n",
       " 'got',\n",
       " 'pap',\n",
       " 'tel',\n",
       " 'kat',\n",
       " 'rom',\n",
       " 'haw',\n",
       " 'ady',\n",
       " 'bcl',\n",
       " 'ngu',\n",
       " 'gos',\n",
       " 'fro',\n",
       " 'lfn',\n",
       " 'kxi',\n",
       " 'mik',\n",
       " 'mdf',\n",
       " 'xmf',\n",
       " 'srn',\n",
       " 'hau',\n",
       " 'sna',\n",
       " 'eus',\n",
       " 'ain',\n",
       " 'fra',\n",
       " 'sma',\n",
       " 'mah',\n",
       " 'fuv',\n",
       " 'drt',\n",
       " 'som',\n",
       " 'chn',\n",
       " 'arq',\n",
       " 'bua',\n",
       " 'sat',\n",
       " 'qxq',\n",
       " 'hin',\n",
       " 'roh',\n",
       " 'div',\n",
       " 'lut',\n",
       " 'ton',\n",
       " 'otk',\n",
       " 'tgk',\n",
       " 'yor',\n",
       " 'nld',\n",
       " 'mnw',\n",
       " 'emx',\n",
       " 'evn',\n",
       " 'ban',\n",
       " 'ibo',\n",
       " 'ike',\n",
       " 'hoc',\n",
       " 'osp',\n",
       " 'lld',\n",
       " 'gle',\n",
       " 'hil',\n",
       " 'kin',\n",
       " 'tly',\n",
       " 'csb',\n",
       " 'mrj',\n",
       " 'dsb',\n",
       " 'hye',\n",
       " 'bak',\n",
       " 'sux',\n",
       " 'tgl',\n",
       " 'izh',\n",
       " 'gsw',\n",
       " 'snd',\n",
       " 'crh',\n",
       " 'zul',\n",
       " 'mwl',\n",
       " 'mri',\n",
       " 'tah',\n",
       " 'kpv',\n",
       " 'nap',\n",
       " 'syl',\n",
       " 'udm',\n",
       " 'war',\n",
       " 'kal',\n",
       " 'prg',\n",
       " 'lat',\n",
       " 'mww',\n",
       " 'ewe',\n",
       " 'oar',\n",
       " 'sme',\n",
       " 'sgs',\n",
       " 'glv',\n",
       " 'que',\n",
       " 'cym',\n",
       " 'gla',\n",
       " 'cho',\n",
       " 'urh',\n",
       " 'spa',\n",
       " 'crs',\n",
       " 'ces',\n",
       " 'ceb',\n",
       " 'nlv',\n",
       " 'ksh',\n",
       " 'kum',\n",
       " 'est',\n",
       " 'ajp',\n",
       " 'kan',\n",
       " 'cjy',\n",
       " 'gbm',\n",
       " 'nst',\n",
       " 'knc',\n",
       " 'kiu',\n",
       " 'guj',\n",
       " 'hdn',\n",
       " 'cha',\n",
       " 'ara',\n",
       " 'ngt',\n",
       " 'aym',\n",
       " 'lit',\n",
       " 'laa',\n",
       " 'bel',\n",
       " 'mus',\n",
       " 'avk']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_langs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "def language_detection_textcleaning(string):\n",
    "    string = re.sub(\n",
    "        'http\\\\S+|www.\\\\S+',\n",
    "        '',\n",
    "        ' '.join(\n",
    "            [i for i in string.split() if i.find('#') < 0 and i.find('@') < 0]\n",
    "        ),\n",
    "    )\n",
    "\n",
    "    chars = ',.()!:\\'\"/;=-'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "    string = string.replace('\\n', '').replace('\\t', '')\n",
    "\n",
    "    string = re.sub(\n",
    "        '[0-9!@#$%^&*()_\\\\-+{}|\\\\~`\\'\";:?/.>,<]', ' ', string, flags=re.UNICODE\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "\n",
    "    return string.lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████| 9479117/9479117 [00:48<00:00, 197312.55it/s]\n"
     ]
    }
   ],
   "source": [
    "others = [language_detection_textcleaning(s) for s in tqdm(others)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "others = list(set(others))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9302198"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████| 79693/79693 [00:00<00:00, 354432.00it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "79664"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chinese = [language_detection_textcleaning(s) for s in tqdm(cmn + yue)]\n",
    "chinese = list(set(chinese))\n",
    "len(chinese)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('chinese-standard.jsonl.dedup', 'w') as fopen:\n",
    "    for s in chinese:\n",
    "        fopen.write(f'{json.dumps(s)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|████▋                       | 1577855/9302198 [00:01<00:05, 1408655.98it/s]"
     ]
    }
   ],
   "source": [
    "with open('others.jsonl.dedup', 'w') as fopen:\n",
    "    for s in tqdm(others):\n",
    "        fopen.write(f'{json.dumps(s)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
